{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.8.5 64-bit",
   "metadata": {
    "interpreter": {
     "hash": "f18a4039b52cba542189536e87fe9a35f2c3ec28f83d49c4da45938af89bff90"
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import argparse\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "# TODO: Move utils into separate file\n",
    "\"\"\"Start of Utils\"\"\"\n",
    "def defect_codes_to_analyze(category=\"all\"):\n",
    "    \"\"\"\n",
    "    Returns a list of defect codes of a particular defect category that you want to analyze\n",
    "    The defect codes belonging to different categories\n",
    "    \"\"\"\n",
    "    deposit_codes = [\n",
    "        \"DAE\",\n",
    "        \"DAGS\",\n",
    "        \"DAR\",\n",
    "        \"DAZ\",\n",
    "        \"DSV\",\n",
    "        \"DSGV\",\n",
    "        \"DSC\",\n",
    "        \"DSZ\",\n",
    "        \"DNF\",\n",
    "        \"DNGV\",\n",
    "        \"DNZ\",\n",
    "    ]\n",
    "    deformed_codes = [\"DR\", \"DFBR\", \"DFBI\", \"DFC\", \"DFE\", \"DTBR\", \"DTBI\"]\n",
    "    infiltration_codes = [\n",
    "        \"IS\",\n",
    "        \"ISB\",\n",
    "        \"ISJ\",\n",
    "        \"ISC\",\n",
    "        \"ISL\",\n",
    "        \"IW\",\n",
    "        \"IWB\",\n",
    "        \"IWC\",\n",
    "        \"IWJ\",\n",
    "        \"IWL\",\n",
    "        \"ID\",\n",
    "        \"IDB\",\n",
    "        \"IDC\",\n",
    "        \"IDJ\",\n",
    "        \"IDL\",\n",
    "        \"IR\",\n",
    "        \"IRB\",\n",
    "        \"IRC\",\n",
    "        \"IRJ\",\n",
    "        \"IRL\",\n",
    "        \"IG\",\n",
    "        \"IGB\",\n",
    "        \"IGC\",\n",
    "        \"IGL\",\n",
    "        \"IGJ\",\n",
    "    ]\n",
    "    hole_codes = [\"H\", \"HSV\", \"HVV\"]\n",
    "    fracture_codes = [\"FL\", \"FC\", \"FM\", \"FS\", \"FH\", \"FH2\", \"FH3\", \"FH4\"]\n",
    "    crack_codes = [\"CL\", \"CC\", \"CM\", \"CS\", \"CH\", \"CH2\", \"CH3\", \"CH4\"]\n",
    "    broken_codes = [\"B\", \"BSV\", \"BVV\"]\n",
    "    collapse_codes = [\"X\"]\n",
    "\n",
    "    tap_codes = [\n",
    "        \"TB\",\n",
    "        \"TBI\",\n",
    "        \"TBD\",\n",
    "        \"TBC\",\n",
    "        \"TBA\",\n",
    "        \"TF\",\n",
    "        \"TFI\",\n",
    "        \"TFD\",\n",
    "        \"TFC\",\n",
    "        \"TFA\",\n",
    "        \"TFB\",\n",
    "        \"TR\",\n",
    "        \"TRI\",\n",
    "        \"TRD\",\n",
    "        \"TRC\",\n",
    "        \"TRA\",\n",
    "        \"TRB\",\n",
    "        \"TS\",\n",
    "        \"TSI\",\n",
    "        \"TSD\",\n",
    "        \"TSA\",\n",
    "        \"TSB\",\n",
    "    ]\n",
    "    root_codes = [\n",
    "        \"RFB\",\n",
    "        \"RFL\",\n",
    "        \"RFC\",\n",
    "        \"RFJ\",\n",
    "        \"RMB\",\n",
    "        \"RML\",\n",
    "        \"RMC\",\n",
    "        \"RMJ\",\n",
    "        \"RBB\",\n",
    "        \"RBL\",\n",
    "        \"RBC\",\n",
    "        \"RBJ\",\n",
    "        \"RTB\",\n",
    "        \"RTL\",\n",
    "        \"RTC\",\n",
    "        \"RTJ\",\n",
    "    ]\n",
    "    joint_offset_codes = [\n",
    "        \"JOS\",\n",
    "        \"JOM\",\n",
    "        \"JOL\",\n",
    "        \"JOSD\",\n",
    "        \"JOMD\",\n",
    "        \"JOLD\",\n",
    "        \"JSS\",\n",
    "        \"JSM\",\n",
    "        \"JSL\",\n",
    "        \"JAS\",\n",
    "        \"JAM\",\n",
    "        \"JAL\",\n",
    "    ]\n",
    "\n",
    "    defects_all = (\n",
    "        deposit_codes\n",
    "        + deformed_codes\n",
    "        + infiltration_codes\n",
    "        + hole_codes\n",
    "        + fracture_codes\n",
    "        + crack_codes\n",
    "        + broken_codes\n",
    "        + root_codes\n",
    "        + joint_offset_codes\n",
    "        + collapse_codes\n",
    "    )\n",
    "    defects_struct = (\n",
    "        deformed_codes\n",
    "        + hole_codes\n",
    "        + fracture_codes\n",
    "        + crack_codes\n",
    "        + broken_codes\n",
    "        + joint_offset_codes\n",
    "        + collapse_codes\n",
    "    )\n",
    "    defects_operat = root_codes + deposit_codes\n",
    "\n",
    "    if category == \"all\":\n",
    "        return defects_all\n",
    "    elif category == \"structural\":\n",
    "        return defects_struct\n",
    "    elif category == \"operational\":\n",
    "        return defects_operat\n",
    "\n",
    "    else:\n",
    "        raise ValueError(\n",
    "            \"Incorrect input. Category should be all, structural, or operational\"\n",
    "        )\n",
    "\n",
    "\"\"\"End of Utils\"\"\"\n",
    "\n",
    "def get_pacp_grade(defect_code):\n",
    "    grades = {\n",
    "        \"JOM\": 1,\n",
    "        \"JOL\": 2,\n",
    "        \"JOMD\": 1,\n",
    "        \"JOLD\": 2,\n",
    "        \"JSM\": 1,\n",
    "        \"JSL\": 2,\n",
    "        \"JAM\": 1,\n",
    "        \"JAL\": 2,\n",
    "        \"X\": 5,\n",
    "        \"B\": 4,\n",
    "        \"BSV\": 5,\n",
    "        \"BVV\": 5,\n",
    "        \"DR\": 5,\n",
    "        \"DFBR\": 5,\n",
    "        \"DFBI\": 5,\n",
    "        \"DFC\": 5,\n",
    "        \"DFE\": 5,\n",
    "        \"DTBR\": 5,\n",
    "        \"DTBI\": 5,\n",
    "        \"H\": 4,\n",
    "        \"HSV\": 5,\n",
    "        \"HVV\": 5,\n",
    "        \"FL\": 3,\n",
    "        \"FC\": 2,\n",
    "        \"FM\": 4,\n",
    "        \"FS\": 3,\n",
    "        \"FH2\": 4,\n",
    "        \"FH3\": 5,\n",
    "        \"FH4\": 5,\n",
    "        \"CL\": 2,\n",
    "        \"CC\": 1,\n",
    "        \"CM\": 3,\n",
    "        \"CS\": 2,\n",
    "        \"CH2\": 4,\n",
    "        \"CH3\": 5,\n",
    "        \"CH4\": 5,\n",
    "    }\n",
    "    try:\n",
    "        return grades[defect_code]\n",
    "    except ValueError:\n",
    "        print(\"Defect code not in dict\")\n",
    "\n",
    "\n",
    "def filter_df_by_defects(df_cond, keep_defects):\n",
    "    \"\"\"Delete defects that are not in keep_defects\"\"\"\n",
    "    print(\n",
    "        f\"Total number of inspections to begin with are {format(df_cond['InspectionID'].nunique())}\"\n",
    "    )\n",
    "    df_cond = df_cond[df_cond['PACP_Code'].isin(keep_defects)]\n",
    "    print(\n",
    "        f\"Number of inspections with defects under consideration are: {df_cond['InspectionID'].nunique()}\"\n",
    "    )\n",
    "\n",
    "    return df_cond\n",
    "\n",
    "\n",
    "def identify_clusters_in_single_inspection(df_cond, cluster_dist_thresh, insp_id):\n",
    "    \"\"\"\n",
    "    Two defects are considered to be in a cluster if they are <=3 feet apart from one another\n",
    "    \n",
    "    Returns empy list if no clusters were identified\n",
    "    \"\"\"\n",
    "    clusters = []\n",
    "    df_temp = df_cond.copy(deep=True)\n",
    "    df_temp = df_temp.sort_values(by=[\"Distance\"])\n",
    "\n",
    "    indices = df_temp.index\n",
    "    defect_prev, defect_curr = \"\", \"\"\n",
    "    dist_prev, dist_curr = 0, 0\n",
    "    cluster_curr = []\n",
    "\n",
    "    for index in indices:\n",
    "        defect_curr = df_temp.at[index, \"PACP_Code\"]  # Defect code at current index\n",
    "        cond_id = df_temp.at[index, \"ConditionID\"]  # ConditionID at current index\n",
    "        video_frame = df_temp.at[index, \"Counter\"]  # Frame at current index\n",
    "\n",
    "        dist_curr = float(\n",
    "            df_temp.at[index, \"Distance\"]\n",
    "        )  # Distance of defect at current index\n",
    "\n",
    "        if abs(dist_curr - dist_prev) >= cluster_dist_thresh:\n",
    "            clusters.append(cluster_curr)\n",
    "            cluster_curr = []\n",
    "        \n",
    "        cluster_curr.append((insp_id, defect_curr, dist_curr, cond_id, video_frame))\n",
    "\n",
    "        dist_prev = dist_curr\n",
    "        defect_prev = defect_curr\n",
    "\n",
    "    return clusters\n",
    "\n",
    "\n",
    "def identify_clusters_in_multiple_inspections(df_cond, cluster_dist_thresh):\n",
    "    clusters = []\n",
    "    insp_ids = df_cond[\"InspectionID\"].unique()\n",
    "\n",
    "    # Loop through all inspections\n",
    "    for insp_id in insp_ids:\n",
    "        # Get df corresponding to a particular inspection\n",
    "        df_cond_single_inspection = df_cond[df_cond[\"InspectionID\"] == insp_id]\n",
    "        # Identify clusters in inspection and add to list of clusters\n",
    "        clusters.extend(identify_clusters_in_single_inspection(df_cond_single_inspection, cluster_dist_thresh, insp_id))\n",
    "\n",
    "    # Delete empty clusters\n",
    "    clusters = list(filter(lambda a: a != [], clusters))\n",
    "    return clusters\n",
    "\n",
    "\n",
    "def calc_num_clusters(clusters):\n",
    "    num_clusters = defaultdict(int)\n",
    "    for cluster in clusters:\n",
    "        if len(cluster) >= 1:\n",
    "            num_clusters[len(cluster)] += 1\n",
    "\n",
    "    return num_clusters, max_cluster_len\n",
    "\n",
    "\n",
    "def calc_cluster_severity(cluster, len_thresh = 3):\n",
    "    cluster_length = 0\n",
    "    grade = 0\n",
    "\n",
    "    for _, defect_code, _, _, _ in cluster:\n",
    "        grade += get_pacp_grade(defect_code)\n",
    "\n",
    "    num_defects = len(cluster)\n",
    "    cluster_length = (\n",
    "        cluster[num_defects - 1][2] - cluster[0][2]\n",
    "    )  # Length is distance of last - first\n",
    "    if cluster_length < len_thresh:\n",
    "        cluster_length = len_thresh\n",
    "\n",
    "    severity = grade / cluster_length\n",
    "    return severity, grade\n",
    "\n",
    "\n",
    "def filter_clusters(clusters, num_defects_thresh, severity_thresh):\n",
    "    \"\"\"Filter clusters by number of defects and severity\"\"\"\n",
    "    filtered_clusters = [\n",
    "            cluster for cluster in clusters if len(cluster) >= num_defects_thresh and calc_cluster_severity(cluster)[0] > severity_thresh\n",
    "        ]\n",
    "    return filtered_clusters\n",
    "\n",
    "def save_clusters_as_csv(filtered_clusters):\n",
    "    res = []\n",
    "    for cluster in filtered_clusters:\n",
    "        str_defects_in_cluster = \"\"\n",
    "\n",
    "        for _, defect_code, distance, _, _ in cluster:\n",
    "            str_defects_in_cluster+=f\"{defect_code} @ {distance}ft\\n\"\n",
    "        \n",
    "        # Get the video sid from the first defect in the cluster\n",
    "        vid_sid = cluster[0][0]\n",
    "        # Get timestamp of first defect in the cluster\n",
    "        time_stamp = round(cluster[0][4]/30, 1)\n",
    "        \n",
    "        url = f\"https://pioneer.sewerai.com/dashboard/review/{vid_sid}?t={time_stamp}\"\n",
    "\n",
    "        res.append({\n",
    "            'Video SID': vid_sid,\n",
    "            'Defects in Cluster': str_defects_in_cluster,\n",
    "            'PIONEER URL': url\n",
    "        })\n",
    "\n",
    "    df = pd.DataFrame(res)\n",
    "    df.to_excel(\"clusters_in_dataset.xlsx\", index=False)\n",
    "\n",
    "\n",
    "def parse_args():\n",
    "    parser = argparse.ArgumentParser(description=\"\")\n",
    "    parser.add_argument(\n",
    "        \"--cond_db\", help=\"Path to CSV file containing PACP condition database\"\n",
    "    )\n",
    "    parser.add_argument(\n",
    "        \"--defect_category\", help=\"Choose between: all, structural, and operational\"\n",
    "    )\n",
    "    parser.add_argument(\"--cluster_dist_thresh\", help=\"Cluster threshold distance\")\n",
    "    args = parser.parse_args()\n",
    "    return args\n",
    "\n",
    "def ipynb_fake_args(cond_db = \"data/Condition_Databases/Conditions_SAI.csv\", defect_category = \"structural\", cluster_dist_thresh = 6.0):\n",
    "    \"\"\"\n",
    "    cluster_dist_thresh: maximum distance between 2 defects to consider them in a cluster\n",
    "    \"\"\"\n",
    "    class Args:\n",
    "        pass\n",
    "    args = Args()\n",
    "    args.cond_db = cond_db\n",
    "    args.defect_category = defect_category\n",
    "    args.cluster_dist_thresh = cluster_dist_thresh\n",
    "    return args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Total number of inspections to begin with are 35\nNumber of inspections with defects under consideration are: 8\nNumber of clusters of different sizes is: defaultdict(<class 'int'>, {4: 3, 3: 1})\n"
     ]
    }
   ],
   "source": [
    "# main\n",
    "is_notebook = True\n",
    "if not is_notebook:\n",
    "    args = parse_args()\n",
    "else:\n",
    "    args = ipynb_fake_args()\n",
    "\n",
    "# Read csv containing single or multiple inspections\n",
    "df_cond = pd.read_csv(args.cond_db, sep=\",\")\n",
    "\n",
    "# Get list of defect codes to keep\n",
    "keep_defects = defect_codes_to_analyze(args.defect_category)\n",
    "\n",
    "# Filter the df to only keep particular code\n",
    "df_cond = filter_df_by_defects(df_cond, keep_defects)\n",
    "\n",
    "# Identify clusters\n",
    "clusters = identify_clusters_in_multiple_inspections(df_cond, int(args.cluster_dist_thresh))\n",
    "\n",
    "filtered_clusters = filter_clusters(clusters, num_defects_thresh=3, severity_thresh=1)\n",
    "\n",
    "# Calculate number of clusters\n",
    "num_clusters, max_cluster_len = calc_num_clusters(filtered_clusters)\n",
    "print(f\"Number of clusters of different sizes is: {num_clusters}\")\n",
    "\n",
    "# Save clusters as csv\n",
    "save_clusters_as_csv(filtered_clusters)\n",
    "\n",
    "# TODO: Visualize defect clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "'\\nExample usage:\\n\\npython defect_cluster_identifier.py --cond_db \"data/PACP_databases/Conditions_Hazen_Sawyer.csv\" --defect_category \"structural\" --num_defects_thresh 3\\n\\n'"
      ]
     },
     "metadata": {},
     "execution_count": 115
    }
   ],
   "source": [
    "\n",
    "\"\"\"\n",
    "Example usage:\n",
    "\n",
    "python defect_cluster_identifier.py --cond_db \"data/Condition_Databases/Conditions_SAI.csv\" --defect_category \"structural\" --num_defects_thresh 3\n",
    "\n",
    "\"\"\""
   ]
  }
 ]
}